# -*- coding: utf-8 -*-
from scrapy import Spider, Request
from ccgpsax.items import CcgpsaxItem, ProvineListItem, CityListItem
import re
import chardet

class CcgpSpider(Spider):
    name = 'ccgp'
    allowed_domains = ['www.ccgp-shaanxi.gov.cn']
    start_urls = ['http://www.ccgp-shaanxi.gov.cn']

    start_page = 0
    fetch_page = 3

    def start_requests(self):
        yield Request(url=self.start_urls[0], callback=self.index_request)

    def index_request(self, response):
        provineUrl = response.css('.center_center_k2:first-child .bb1:first-child > tr:first-child > td:first-child > a::attr(href)').extract_first()
        cityTempUrl = response.css('.center_center_k2')
        cityUrl = cityTempUrl[1].css('.bb1:first-child > tr:first-child > td:first-child > a::attr(href)').extract_first()

        yield Request(response.urljoin(provineUrl), self.provine_request)
        yield Request(response.urljoin(cityUrl), self.city_request)

    def provine_request(self, response):
        list = response.css('.tab > tr:nth-child(2) > td:first-child > table > tr:first-child > td:first-child > table > tr')
        nextPage = response.css('form > font > a:nth-last-child(2)::attr(href)').extract_first()

        # 获取下一页的页码
        pageNumber = re.findall(r'pages=(.*)', nextPage)

        for cell in list:
            item = ProvineListItem()
            item['title'] = cell.css('td:first-child > a::text').extract_first().strip()
            item['url'] = response.urljoin(cell.css('td:first-child > a::attr(href)').extract_first())
            item['date'] = re.sub('[\[|\]]', '', cell.css('td:last-child::text').extract_first())

            # TODO:从招聘名称中筛选法务、法律咨询、事务所三个字样
            yield item

        if nextPage and int(pageNumber[0]) <= 3:
            yield Request(url=response.urljoin(nextPage), callback=self.provine_request)

    def city_request(self, response):
        list = response.css('.tab > tr:nth-child(2) > td:first-child > table > tr:first-child > td:first-child > table > tr')
        nextPage = response.css('form > font > a:nth-last-child(2)::attr(href)').extract_first()

        # 获取下一页的页码
        pageNumber = re.findall(r'pages=(.*)', nextPage)

        for cell in list:
            item = CityListItem()
            item['title'] = cell.css('td:first-child > a::text').extract_first().strip()
            item['url'] = response.urljoin(cell.css('td:first-child > a::attr(href)').extract_first())
            item['date'] = re.sub('[\[|\]]', '', cell.css('td:last-child::text').extract_first())

            # TODO:从招聘名称中筛选法务、法律咨询、事务所三个字样
            yield item

        if nextPage and int(pageNumber[0]) <= 3:
            yield Request(url=response.urljoin(nextPage), callback=self.provine_request)

    def parse(self, response):
        pass
